import hdfs
import csv
import os
import time
from db_class.mongdbClass import MongdbClass

class HDFSClass():
    def __init__(self, host, head):
        self.fname = f"{str(int(time.time()))}.csv"
        self.f = open(self.fname, mode='a', encoding='utf-8')
        self.writer = csv.writer(self.f)
        self.writer.writerow(head)
        self.client = hdfs.Client(host)

    def callback(self, filename, size):
        print(filename, "完成一个chunk上传", "当前大小", size)
        if size == -1:
            print("文件上传成功")

    # 写入本地 csv 文件
    def writeFile(self, data):
        new_data = []
        for item in data:
            new_data.append((
                item.get("title", ""),
                item.get("type", ""),
                item.get("time", ""),
                item.get("url", ""),
                item.get("date", ""),
                item.get("month", ""),
                item.get("year", ""),
            ))
        self.writer.writerows(new_data)

    # 上传文件到 hdfs
    def uploadFile(self):
        self.client.upload(local_path=self.fname, hdfs_path="/user/caixinpeng/news", progress=self.callback, cleanup=True)
        os.remove(self.fname)

if __name__ == "__main__":
    mongo = MongdbClass("182.92.148.234", "Cache", "cache")
    data = []
    for i in mongo.findAll():
        data.append((i["title"], i["type"], i['time'], i["url"],i["date"], i["month"], i['year']))

    db = HDFSClass("http://master:9870/", ("title", "type", "time", "url", "date", "month", "year"))
    db.writeFile(data)
    db.uploadFile()